package com.lheia.process


import com.lheia.jcb.spider.ApprovalPublicity
import com.lheia.jcb.spider.IpeSpiderData
import com.lheia.tool.ToolDateTime
import com.lheia.tool.ToolUserAgents
import com.lheia.util.PageFieldConstants
import com.lheia.util.RequestExtraConstants
import com.lheia.util.ToolUrl
import us.codecraft.webmagic.Page
import us.codecraft.webmagic.Request
import us.codecraft.webmagic.Site
import us.codecraft.webmagic.processor.PageProcessor
import us.codecraft.webmagic.selector.HtmlNode

/**
 * 广州黄埔区爬虫
 */
class SpierGZHPProcess implements PageProcessor{
    private Site site = Site.me().setRetryTimes(3).setUserAgent(ToolUserAgents.userAgentRandom())
    private String province = "广东省"
    private String city = "广州市"
    private String county = "黄浦区"
    private String approveDept = "黄浦区环保局"

    void setIfTest(boolean ifTest) {
        this.ifTest = ifTest
    }
    protected boolean ifTest = true
    /**
     * process the page, extract urls to fetch, extract the data and store
     *
     * @param page page
     */
    @Override
    /**
     * 主要思路使用xpath解析列表也对象添加到请求队列中
     *
     */
    void process(Page page) {
        def methodName = page.getRequest().getExtra(RequestExtraConstants.CALLBACK)
        if(methodName){
            this."$methodName"(page)
        }
        processTargetResquest(page)
    }

    /***
     * 列表处理
     * @param page
     */
    void processList(Page page){
        /**列表node查找***/
        HtmlNode liList = page.getHtml().xpath("//div[@class='news_list']//li")
        liList.getElements().each{
            def listNode = it.getElementsByTag("a").get(0)
            def detailUrl = listNode.attr("href")
            def title = listNode.text()

            /***
             * 过滤title是否抓取抓取之后
             */
            def abs= ToolUrl.getAbsoluteURL(page.getUrl().toString(),detailUrl)
            def detailRequest = new Request(abs)
            if(title.contains("建设项目审批") || title.contains("验收")){
                println String.join(",",[title,abs,it.getElementsByTag("span").get(0).text()])
                detailRequest.putExtra(RequestExtraConstants.CALLBACK,"processDetailsHP")
                page.addTargetRequest(detailRequest)
            }else if(title.contains("处罚")){
                println String.join(",",[title,abs,it.getElementsByTag("span").get(0).text()])
                detailRequest.putExtra(RequestExtraConstants.CALLBACK,"processDetailsJG")
                page.addTargetRequest(detailRequest)
            }

        }
        page.getResultItems().setSkip(true)
        processPaging(page)
    }
    /**
     * 分页处理
     * @param page
     */
    void processPaging(Page page){
        def nextPageNode = page.getHtml().getDocument().getElementsContainingOwnText("下一页")
        /**
         * 获取分页标签
         */
        if(nextPageNode.size()>0){
            def url = nextPageNode.get(0).attr("href")
            def abs= ToolUrl.getAbsoluteURL(page.getUrl().toString(),url)
            def pageRequest = new Request(abs)
            /***
             * 分页请求不需要进行去重
             */
            pageRequest.putExtra(RequestExtraConstants.NO_NEED_TO_REMOVE,true)
            /**
             * 是否使用phantomJS
             */
            pageRequest.putExtra(RequestExtraConstants.PHANTOM_JS,page.getRequest().getExtra(RequestExtraConstants.PHANTOM_JS))
            /**
             * 处理回调
             */
            pageRequest.putExtra(RequestExtraConstants.CALLBACK,"processList")
            println("下一页,"+pageRequest)
            page.addTargetRequest(pageRequest)
        }

    }
    /**
     * 竣工验收详情页处理
     * @param page
     */
    def processDetailsJG(Page page){
        def resList = []
        def doc = page.getHtml().getDocument()
        def dataMap = [:]
        dataMap.regionName  = this.province
        dataMap.regionCode =this.city
        dataMap.county = this.county
        dataMap.approveDept = this.approveDept

        /**获取标题**/
        def htmlHead = doc.select("div.content_xxgkAttr").select("li")
        dataMap.pubDate = ToolDateTime.extractDate(htmlHead.get(3).toString())
        dataMap.title = doc.select("h1.content_title")
        dataMap.spiderFileUrl = page.getUrl()

        /**获取列表内容**/
        def trList = doc.select("table").select("tr:gt(1)")
        for(def trNode:trList ){
            def tdList = trNode.select("td")
            dataMap.projectName = tdList.get(0).text()
            dataMap.approveDate = ToolDateTime.extractDate(tdList.get(1).text())
            def  approvalPublicity = new ApprovalPublicity()
            approvalPublicity.properties  = dataMap
            resList << approvalPublicity
        }
        page.putField(PageFieldConstants.RESULT_DATA,resList)
    }

    def processDetailsCF(Page page){
        def document = page.getHtml().getDocument()
        def resList = []
        def ipeSpiderData = [:]
        ipeSpiderData.province  = "广东省"
        ipeSpiderData.city = "广州市"
        ipeSpiderData.county = "黄埔区"
        ipeSpiderData.pubOrg = "黄浦区环保局"
        def htmlHead = document.select("div.content_xxgkAttr").select("li")
        ipeSpiderData.timeOccurrence = ToolDateTime.extractDate(htmlHead.get(3).toString())
//        publicSentiment.title = doc.select("h1.content_title")
        ipeSpiderData.spiderFileUrl = document.baseUri()
        ipeSpiderData.status = "行政处罚"
        def trList = document.select("table").select("tr:gt(1)")
        for(def trNode:trList ){
            def tdList = trNode.select("td")
            ipeSpiderData.illegalDetail = trNode.html()
            ipeSpiderData.companyName = tdList.get(1).text()
            ipeSpiderData.messageSource =tdList.get(3).text()
            def fileUrl =tdList.get(9).select("a").attr("href")
            if(fileUrl){
                ipeSpiderData.fileUrl = ToolUrl.getAbsoluteURL(document.baseUri(),fileUrl)
            }

            def resData = new IpeSpiderData()
            resData.properties = ipeSpiderData

            resList << resData
        }
        page.putField(PageFieldConstants.RESULT_DATA,resList)
    }
    /**
     * 建设项目
     */
    def processDetailsHP(Page page){
        def resList = []
        def doc = page.getHtml().getDocument()
        def dataMap = [:]
        dataMap.regionName  = this.province
        dataMap.regionCode =this.city
        dataMap.county = this.county
        dataMap.approveDept = this.approveDept

        /**获取标题**/
        def htmlHead = doc.select("div.content_xxgkAttr").select("li")
        dataMap.pubDate = ToolDateTime.extractDate(htmlHead?.get(3).toString())
        dataMap.title = doc.select("h1.content_title")
        dataMap.spiderFileUrl = page.getUrl()

        /**获取列表内容**/
        def trList = doc.select("table").select("tr:gt(1)")
        for(def trNode:trList ){
            def tdList = trNode.select("td")
            dataMap.projectName = tdList.get(0).text()
            dataMap.approveDate = ToolDateTime.extractDate(tdList.get(1).text())
            def  approvalPublicity = new ApprovalPublicity()
            approvalPublicity.properties  = dataMap
            resList << approvalPublicity
        }
        page.putField(PageFieldConstants.RESULT_DATA,resList)
    }
    /**
     * 如果为测试环境或者开发环境不计入redis去重只使用redisList队列
     * @param page
     */
    void processTargetResquest(Page page){
        if(ifTest){
            page.getTargetRequests().each {
                it.putExtra(RequestExtraConstants.NO_NEED_TO_REMOVE,true)
            }
        }
    }


    /**
     * get the site settings
     *
     * @return site* @see Site
     */
    @Override
    Site getSite() {
        return site
    }
}
